package cn.com.duiba.millionaire.center.spider.spdierjob;

import cn.com.duiba.millionaire.center.spider.common.PageConsumer;
import cn.com.duiba.millionaire.center.spider.entity.SpiderAppInfoEntity;
import cn.com.duiba.millionaire.center.spider.service.SpiderAppInfoService;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.selector.Html;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

/**
 * Created by sunchangji on 2018/6/6.
 */
@Component
public class WandoujiaSpider {

    @Autowired
    private SpiderAppInfoService spiderAppInfoService;

    private static String DEST_PRIX = "http://www.wandoujia.com/apps-";


    public PageConsumer pageConsumer = page -> {
        String url = page.getUrl().get();
        if(url.contains("http://www.wandoujia.com/category")){
            List<String> targetUrlList = page.getHtml().links().regex("http://www.wandoujia.com/apps-[0-9a-zA-Z.]*").all();
            if(CollectionUtils.isNotEmpty(targetUrlList)){
                Set<String> targetUrls = Sets.newHashSet(targetUrlList);
                String pageIndex = StringUtils.substringAfterLast(url,"/");
                if(StringUtils.isNotEmpty(pageIndex)){
                    Integer index = Integer.valueOf(pageIndex)+1;
                    targetUrls.add(url.substring(0,url.lastIndexOf("/")+1)+index);
                }
                page.addTargetRequests(Lists.newArrayList(targetUrls));
            }
        }else if(url.contains(DEST_PRIX)){
            Html html = page.getHtml();
            //app-name
            String appName = html.xpath("//div/p/span[@class='title']/text()").get();
            if(StringUtils.isNotEmpty(appName)){
                List<String> category = html.xpath("//dd[@class='tag-box']/a/text()").all();
                String updateTime = html.xpath("//time[@id='baidu_time']/text()").get();
                String company = html.xpath("//dd/span[@class='dev-sites']/text()").get();
                List<String> descList = html.xpath("//div[@class='desc-info']/div/div/p/text()").all();
                List<String> descreption = new ArrayList<>();
                for(String desc : descList){
                    descreption.add(StringUtils.deleteWhitespace(desc));
                }
                SpiderAppInfoEntity entity = new SpiderAppInfoEntity();
                entity.setAppName(appName);
                entity.setCategory(category == null? null : Joiner.on(",").join(category));
                entity.setCompany(company);
                entity.setUpdatedDate(updateTime);
                entity.setEmail(url);
                entity.setDescription(CollectionUtils.isEmpty(descreption) ? null : Joiner.on(",").join(descreption));
                spiderAppInfoService.inserAppInfo(entity);
            }

        }

    };

}
